In [1]:
# Computations
import pandas as pd
import numpy as np
from sklearn.metrics import confusion_matrix, roc_curve, auc, classification_report, plot_confusion_matrix
from sklearn.model_selection import train_test_split, GridSearchCV, RandomizedSearchCV, cross_val_score, KFold
# sklearn
from sklearn.metrics import classification_report, accuracy_score, f1_score, precision_score, recall_score
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split, GridSearchCV, RandomizedSearchCV, cross_val_score
from sklearn import metrics
from sklearn.feature_selection import RFE
from sklearn.utils.fixes import loguniform
from sklearn.model_selection import KFold
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier
import xgboost as xgb
from sklearn.ensemble import VotingClassifier
# preprocessing
from sklearn.preprocessing import StandardScaler
# Visualisation libraries
## Text
from colorama import Fore, Back, Style
from IPython.display import Image, display, Markdown, Latex
## seaborn
import seaborn as sns
sns.set_context("paper", rc={"font.size":12,"axes.titlesize":14,"axes.labelsize":12})
sns.set_style("white")
## matplotlib
import matplotlib.pyplot as plt
from matplotlib.patches import Ellipse, Polygon
import matplotlib.gridspec as gridspec
import matplotlib.colors
from pylab import rcParams
plt.style.use('seaborn-whitegrid')
plt.rcParams['figure.figsize'] = 14, 8
plt.rcParams['axes.labelsize'] = 14
plt.rcParams['xtick.labelsize'] = 12
plt.rcParams['ytick.labelsize'] = 12
plt.rcParams['text.color'] = 'k'
%matplotlib inline
## plotly
from plotly.offline import init_notebook_mode, iplot
import plotly.graph_objs as go
import plotly.offline as py
from plotly.subplots import make_subplots
import plotly.express as px
# Graphics in retina format
%config InlineBackend.figure_format = 'retina'
import warnings
warnings.filterwarnings("ignore")
Anomaly Detection¶
Anomaly detection is a classification process in which rare items, events, or observations in data sets are identified. Learn more about this here. In this article, we investigate Credit Card Fraud Detection dataset from Kaggle.com.
Credit Card Fraud Detection
Context¶
Credit card companies must be able to recognize fraudulent credit card transactions so that customers are not charged for items that they did not purchase.
Content¶
The datasets contain transactions made by credit cards in September 2013 by European cardholders. This dataset presents transactions that occurred in two days, where we have 492 frauds out of 284,807 transactions. The dataset is highly unbalanced, the positive class (frauds) account for 0.172% of all transactions. It contains only numerical input variables which are the result of a PCA transformation. Unfortunately, due to confidentiality issues, we cannot provide the original features and more background information about the data. Features V1, V2, … V28 are the principal components obtained with PCA, the only features which have not been transformed with PCA are 'Time' and 'Amount'. Feature 'Time' contains the seconds elapsed between each transaction and the first transaction in the dataset. The feature 'Amount' is the transaction Amount, this feature can be used for example-dependant cost-sensitive learning. Feature 'Class' is the response variable and it takes value 1 in case of fraud and 0 otherwise.
In [2]:
Data = pd.read_csv('Data/creditcard.csv',sep=',')
Col = []
# Temp = re.findall("(\d+)", s)
for s in Data.columns:
if any(map(str.isdigit, s)) == True:
Temp = s.split('V')
Col.append('V'+ Temp[-1].zfill(2))
else:
Col.append(s)
Data.columns = Col
del Col
display(pd.DataFrame(Data.shape, columns = ['Count'], index = ['Attributes', 'Instances']).T)
In [3]:
Labels = ['Normal', 'Fraud']
Temp = Data['Class'].value_counts(sort = False).to_frame('Count').reset_index()
Temp.columns = ['Class','Count']
Temp['Class'] = Temp['Class'].map(lambda x: Labels[0] if x == 0 else Labels[1])
Temp['Percentage'] = np.round(100* Temp['Count'].values /Temp['Count'].sum(), 2)
display(Temp.style.hide_index().set_precision(2))
fig = px.bar(Temp, y= 'Class', x= 'Percentage', orientation='h', text = 'Count', color_discrete_sequence= ['Bisque'],
height= 220)
fig.update_traces(marker_line_color= 'DarkRed', marker_line_width=1.5, opacity=1)
fig.update_traces(texttemplate='%{text:.2}', textposition='inside')
fig.update_layout(uniformtext_minsize= 8, uniformtext_mode='hide')
fig['layout']['xaxis'].update(range=[0, 100])
fig.update_layout(title = 'Transaction Class Distribution', plot_bgcolor= 'white')
fig.update_xaxes(showline=True, linewidth=1, linecolor='Lightgray', mirror=True)
fig.update_yaxes(showline=True, linewidth=1, linecolor='Lightgray', mirror=True)
fig.show()
As can be seen, nearly, 99.83 percent of the dataset are labeled as Normal.
Number of Transactions vs Amount of Transactions¶
In [4]:
fig, ax = plt.subplots(1, 1, figsize=(16, 6))
_ = ax.hist(Data.loc[Data.Class == 0, 'Amount'], 100, color = '#34495e', hatch = '/', lw = 1.5,
edgecolor = '#3498db', label = Labels[0])
_ = ax.hist(Data.loc[Data.Class == 1, 'Amount'], 10, Color = '#e74c3c', hatch = '\\', lw = 1.5,
edgecolor = 'DarkRed', label = Labels[1])
_ = ax.set_xlabel('Amount')
_ = ax.set_ylabel('Number of Transactions')
_ = ax.set_xlim([0, 2e4])
_ = ax.set_yscale('log')
_ = ax.set_ylim([0, 1e6])
_ = ax.legend(bbox_to_anchor=(1, 1), fontsize=14, ncol=2)
Time vs Amount of Transactions¶
In [5]:
fig, ax = plt.subplots(1, 1, figsize=(16, 6))
_ = ax.scatter(Data.loc[Data.Class == 0, 'Time'], Data.loc[Data.Class == 0, 'Amount'], s= 30,
facecolors='SkyBlue', edgecolors='MidnightBlue', alpha = 0.8, label = Labels[0])
_ = ax.scatter(Data.loc[Data.Class == 1, 'Time'], Data.loc[Data.Class == 1, 'Amount'], s= 30,
facecolors='Orange', edgecolors='DarkRed', alpha = 1, label = Labels[1])
_ = ax.set_xlabel('Time (in seconds)')
_ = ax.set_ylabel('Amount')
_ = ax.set_xlim([-500, Data.Time.max()+500])
_ = ax.set_ylim([-250, 2e4])
_ = ax.legend(bbox_to_anchor=(1, 1), fontsize=14, ncol=2)
Modeling¶
The Dataset is quite large, we would like to use pandas DataFrame sample feature with using a one-tenth of the data as a sample.
In [6]:
df= Data.sample(frac = 0.1, random_state=1)
def Data_info(Inp, Only_NaN = False):
Out = Inp.dtypes.to_frame(name='Data Type').sort_values(by=['Data Type'])
Out = Out.join(Inp.isnull().sum().to_frame(name = 'Number of NaN Values'), how='outer')
Out['Size'] = Inp.shape[0]
Out['Percentage'] = np.round(100*(Out['Number of NaN Values']/Inp.shape[0]),2)
if Only_NaN:
Out = Out.loc[Out['Number of NaN Values']>0]
return Out
display(df.head())
Data_info(df)
Out[6]:
Data Correlations¶
First off, let's define $X$ and $y$ sets.
In [7]:
Target = 'Class'
X = df.drop(columns = [Target])
y = df[Target]
Now, let's take a look at the variance of the features.
In [8]:
display(X.var().sort_values(ascending = False).to_frame(name= 'Variance').T.style.set_precision(2))
As can see some of the variables have high variance and this is not desirable for our modeling. Thus, we would like to standardize features by removing the mean and scaling to unit variance. In this article, we demonstrated the benefits of scaling data using StandardScaler().
In [9]:
Temp = X.columns
X = StandardScaler().fit_transform(X)
X = pd.DataFrame(X, columns = Temp)
del Temp
Correlations of features with Class.
In [10]:
Temp = pd.DataFrame(X, columns = df.drop(columns = [Target]).columns)
Temp[Target] = y
def Correlation_Plot (Df,Fig_Size):
Correlation_Matrix = Df.corr().round(2)
mask = np.zeros_like(Correlation_Matrix)
mask[np.triu_indices_from(mask)] = True
for i in range(len(mask)):
mask[i,i]=0
Fig, ax = plt.subplots(figsize=(Fig_Size,Fig_Size))
sns.heatmap(Correlation_Matrix, ax=ax, mask=mask, annot=True, square=True,
cmap =sns.color_palette("Greens", n_colors=10), linewidths = 0.2, vmin=0, vmax=1, cbar_kws={"shrink": .6})
Correlation_Plot (Temp, 16)
Train and Test Sets¶
In [11]:
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)
pd.DataFrame(data={'Set':['X_train','X_test','y_train','y_test'],
'Shape':[X_train.shape, X_test.shape, y_train.shape, y_test.shape]}).set_index('Set').T
Out[11]:
A number of functions that we will be using throughout this article.
In [12]:
def Performance(clf, X_test = X_test):
df = pd.DataFrame()
y_pred = clf.predict(X_test)
df = df.append({'Score': clf.score(X_test, y_test),
'F1 Score': f1_score(y_test.values, y_pred, average= 'weighted'),
'Precision Score': precision_score(y_test.values, y_pred, average= 'weighted'),
'Recall Score': recall_score(y_test.values, y_pred, average= 'weighted')}, ignore_index=True)
display(df.style.hide_index())
def highlight_max(s):
is_max = s == s.max()
return ['background-color: SpringGreen' if v else '' for v in is_max]
def Feature_Ranking(clf):
df = pd.DataFrame()
for n in range(2, X.shape[1]):
selector = RFE(estimator= clf, n_features_to_select=n, verbose=0)
selector.fit(X_train, y_train)
df = df.append({'Number of Features to Select': n,
'Score':metrics.accuracy_score(y_test, selector.predict(X_test)),
'Features': X.columns[selector.support_].tolist(),
'Best Features':X.columns[selector.ranking_ == 1].tolist()}, ignore_index=True)
df = df[['Number of Features to Select', 'Score', 'Features', 'Best Features']]
display(df.style.apply(highlight_max, subset=['Score']))
return df.loc[df.Score == df.Score.max(), 'Features'].values[0]
def ROC_Curve(clf, X_test = X_test):
# false positive rates, true positive rates and thresholds
fpr, tpr, threshold = metrics.roc_curve(y_test, clf.predict_proba(X_test)[:,1])
fig, ax = plt.subplots(1, 1, figsize=(5.5, 5.5))
_ = ax.plot(fpr, tpr, lw=2, label = 'AUC = %0.2f' % metrics.auc(fpr, tpr))
_ = ax.plot([0, 1], [0, 1],'r--', lw=2)
_ = ax.legend(loc = 'lower right', fontsize = 14)
delta = 0.01
_ = ax.set_xlim([-delta,1+delta])
_ = ax.set_ylim([-delta,1+delta])
_ = ax.set_xlabel('False Positive Rate (FPR)')
_ = ax.set_ylabel('True Positive Rate (TPR)')
Decision Tree Classifier¶
The first classifier that we use here is Decision Tree Classifier.
In [13]:
dtc = DecisionTreeClassifier()
_ = dtc.fit(X_train,y_train)
Performance(dtc)
ROC_Curve(dtc)
However, we could also use RFE from sklearn.feature_selection. This provides feature ranking with recursive feature elimination.
In [14]:
Best_Features = Feature_Ranking(dtc)
Thus, we could use only fewer features and improve the results of classifications. The best features for the classifications are
In [15]:
print(Back.BLACK + Fore.CYAN + Style.NORMAL + 'Best Features:'+ Style.RESET_ALL + ' %s:' % ', '.join(Best_Features))
In [16]:
dtc = DecisionTreeClassifier()
_ = dtc.fit(X_train[Best_Features],y_train)
Performance(dtc, X_test[Best_Features])
ROC_Curve(dtc, X_test[Best_Features])
In [17]:
# Train set
y_pred = dtc.predict(X_train[Best_Features])
Confusion_Matrix = confusion_matrix(y_train, y_pred)
fig, ax = plt.subplots(1, 2, figsize=(15, 5))
fig.suptitle('Train Set', fontsize = 18)
_ = sns.heatmap(Confusion_Matrix, annot=True, annot_kws={"size": 14}, cmap="Blues", ax = ax[0],
linewidths = 0.2, vmin=0, vmax=2e4, cbar_kws={"shrink": 1})
_ = ax[0].set_xlabel('Predicted labels')
_ = ax[0].set_ylabel('True labels');
_ = ax[0].set_title('Confusion Matrix');
_ = ax[0].xaxis.set_ticklabels(Labels)
_ = ax[0].yaxis.set_ticklabels(Labels)
Confusion_Matrix = Confusion_Matrix.astype('float') / Confusion_Matrix.sum(axis=1)[:, np.newaxis]
_ = sns.heatmap(Confusion_Matrix, annot=True, annot_kws={"size": 14}, cmap="Greens", ax = ax[1],
linewidths = 0.2, vmin=0, vmax=1, cbar_kws={"shrink": 1})
_ = ax[1].set_xlabel('Predicted labels')
_ = ax[1].set_ylabel('True labels');
_ = ax[1].set_title('Normalized Confusion Matrix');
_ = ax[1].xaxis.set_ticklabels(Labels)
_ = ax[1].yaxis.set_ticklabels(Labels)
# Test set
y_pred = dtc.predict(X_test[Best_Features])
Confusion_Matrix = confusion_matrix(y_test, y_pred)
fig, ax = plt.subplots(1, 2, figsize=(15, 5))
fig.suptitle('Test Set', fontsize = 18)
_ = sns.heatmap(Confusion_Matrix, annot=True, annot_kws={"size": 14}, cmap="Blues", ax = ax[0],
linewidths = 0.2, vmin=0, vmax=9e3, cbar_kws={"shrink": 1})
_ = ax[0].set_xlabel('Predicted labels')
_ = ax[0].set_ylabel('True labels');
_ = ax[0].set_title('Confusion Matrix');
_ = ax[0].xaxis.set_ticklabels(Labels)
_ = ax[0].yaxis.set_ticklabels(Labels)
Confusion_Matrix = Confusion_Matrix.astype('float') / Confusion_Matrix.sum(axis=1)[:, np.newaxis]
_ = sns.heatmap(Confusion_Matrix, annot=True, annot_kws={"size": 14}, cmap="Greens", ax = ax[1],
linewidths = 0.2, vmin=0, vmax=1, cbar_kws={"shrink": 1})
_ = ax[1].set_xlabel('Predicted labels')
_ = ax[1].set_ylabel('True labels');
_ = ax[1].set_title('Normalized Confusion Matrix');
_ = ax[1].xaxis.set_ticklabels(Labels)
_ = ax[1].yaxis.set_ticklabels(Labels)
RFE can be very useful, especially for cases that the number of features is quite large.
Random Forest Classifier¶
In [18]:
rfc = RandomForestClassifier()
_ = rfc.fit(X_train,y_train)
Performance(rfc)
ROC_Curve(rfc)
In [19]:
Best_Features = Feature_Ranking(rfc)
Thus, we could use only fewer features and improve the results of classifications. The best features for the classifications are
In [20]:
print(Back.BLACK + Fore.CYAN + Style.NORMAL + 'Best Features:'+ Style.RESET_ALL + ' %s:' % ', '.join(Best_Features))
In [21]:
rfc = RandomForestClassifier()
_ = rfc.fit(X_train[Best_Features],y_train)
Performance(rfc, X_test[Best_Features])
ROC_Curve(rfc, X_test[Best_Features])
In [22]:
# Train set
y_pred = rfc.predict(X_train[Best_Features])
Confusion_Matrix = confusion_matrix(y_train, y_pred)
fig, ax = plt.subplots(1, 2, figsize=(15, 5))
fig.suptitle('Train Set', fontsize = 18)
_ = sns.heatmap(Confusion_Matrix, annot=True, annot_kws={"size": 14}, cmap="Blues", ax = ax[0],
linewidths = 0.2, vmin=0, vmax=2e4, cbar_kws={"shrink": 1})
_ = ax[0].set_xlabel('Predicted labels')
_ = ax[0].set_ylabel('True labels');
_ = ax[0].set_title('Confusion Matrix');
_ = ax[0].xaxis.set_ticklabels(Labels)
_ = ax[0].yaxis.set_ticklabels(Labels)
Confusion_Matrix = Confusion_Matrix.astype('float') / Confusion_Matrix.sum(axis=1)[:, np.newaxis]
_ = sns.heatmap(Confusion_Matrix, annot=True, annot_kws={"size": 14}, cmap="Greens", ax = ax[1],
linewidths = 0.2, vmin=0, vmax=1, cbar_kws={"shrink": 1})
_ = ax[1].set_xlabel('Predicted labels')
_ = ax[1].set_ylabel('True labels');
_ = ax[1].set_title('Normalized Confusion Matrix');
_ = ax[1].xaxis.set_ticklabels(Labels)
_ = ax[1].yaxis.set_ticklabels(Labels)
# Test set
y_pred = rfc.predict(X_test[Best_Features])
Confusion_Matrix = confusion_matrix(y_test, y_pred)
fig, ax = plt.subplots(1, 2, figsize=(15, 5))
fig.suptitle('Test Set', fontsize = 18)
_ = sns.heatmap(Confusion_Matrix, annot=True, annot_kws={"size": 14}, cmap="Blues", ax = ax[0],
linewidths = 0.2, vmin=0, vmax=9e3, cbar_kws={"shrink": 1})
_ = ax[0].set_xlabel('Predicted labels')
_ = ax[0].set_ylabel('True labels');
_ = ax[0].set_title('Confusion Matrix');
_ = ax[0].xaxis.set_ticklabels(Labels)
_ = ax[0].yaxis.set_ticklabels(Labels)
Confusion_Matrix = Confusion_Matrix.astype('float') / Confusion_Matrix.sum(axis=1)[:, np.newaxis]
_ = sns.heatmap(Confusion_Matrix, annot=True, annot_kws={"size": 14}, cmap="Greens", ax = ax[1],
linewidths = 0.2, vmin=0, vmax=1, cbar_kws={"shrink": 1})
_ = ax[1].set_xlabel('Predicted labels')
_ = ax[1].set_ylabel('True labels');
_ = ax[1].set_title('Normalized Confusion Matrix');
_ = ax[1].xaxis.set_ticklabels(Labels)
_ = ax[1].yaxis.set_ticklabels(Labels)
It can be seen overall, Random Forest Classifier performed better in this example. Furthermore, using RFE improves the accuracy of this classification.